{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from requests_html import HTMLSession\n",
    "import requests_html\n",
    "import pandas as pd\n",
    "import urllib.parse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "200"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "session = HTMLSession()\n",
    "r = session.get(\"https://www.nfu.edu.cn/tzgg/index.htm\")\n",
    "r.status_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 存\n",
    "with open (\"html_out/_nfu_tzgg.html\", encoding = \"utf8\", mode = \"w\") as fp:\n",
    "    fp.write(r.html.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 读\n",
    "with open (\"html_out/_nfu_tzgg.html\", encoding = \"utf8\", mode = \"r\") as fp:\n",
    "    html_load = fp.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Element html at 0x23fd78029f0>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 解析\n",
    "parsed = requests_html.soup_parse(html_load)\n",
    "parsed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ParseResult(scheme='https', netloc='www.nfu.edu.cn', path='/tzgg/index.htm', params='', query='', fragment='')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 解析\n",
    "base_url = r.url\n",
    "nfu_urlparse = urllib.parse.urlparse(base_url)\n",
    "nfu_urlparse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://www.nfu.edu.cn/tzgg/cd60e06378e54492946c461d5f0574be.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/16fcbd56eab04220b33684acafc1ec66.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/155655d4a7e74c76958fa484c9cc9e24.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/f381db0e5b3e4746b310806a81edbaba.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/ae83ecc6ce894bcb812a17a8fb5fbd7b.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/47ba4c0dfb1443f9945ee3e6df61c921.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/21cd39d341924ffd93228d34c585dc15.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/66fc5e810c664b919bf192de1833c5d1.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/c63acd1ca67746ea93cec0ac93e621b6.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/6e824246669d471ca24b98dfb9538f41.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/df9afab7eb564161bc1a8198b364e79d.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/b57e28f6cd5a4cb7b1e67376809d99cb.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/ad7da9ff14494749b7334b768b2e7207.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/54573ec40397435ca49607cef867b18b.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/e28b2797a45d471b883388659c0d1eb3.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/fc277cccd7f440a2bdadcde84c707815.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/f04e18c7e60b437d8c4591851809edb4.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/4df100c9d8754356a625bd01f70a2d79.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/ed13952ecd4343d99de23d32ea968eb3.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/8eece1bbe3b34e489ebf8afdc9069e0f.htm']"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 重组链接\n",
    "list_URL  = [urllib.parse.urlunparse\\\n",
    "([nfu_urlparse.scheme,nfu_urlparse.netloc,'/'+ nfu_urlparse.path.split('/')[1] +'/' + detail_url,'','',''])\\\n",
    "for detail_url in parsed.xpath('//div[@class=\"news_title\"]/a/@href')]  \n",
    "list_URL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>标题</th>\n",
       "      <th>链接</th>\n",
       "      <th>日期</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>校园管理部关于2021年元旦放假校园生活服务安排的通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/cd60e06378e5449294...</td>\n",
       "      <td>2020-12-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>中山大学南方学院关于2021年元旦放假安排的通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/16fcbd56eab04220b3...</td>\n",
       "      <td>2020-12-17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>关于开展2020年知识产权竞赛的通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/155655d4a7e74c7695...</td>\n",
       "      <td>2020-12-16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>中山大学南方学院关于举办2020年预防艾滋病巡讲活动的通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/f381db0e5b3e4746b3...</td>\n",
       "      <td>2020-12-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>关于开展2020年安全知识竞赛的通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/ae83ecc6ce894bcb81...</td>\n",
       "      <td>2020-12-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>中山大学南方学院关于举办2020～2021学年教职工体育活动的通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/47ba4c0dfb1443f994...</td>\n",
       "      <td>2020-11-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>中山大学南方学院关于举办2020年体育大会的通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/21cd39d341924ffd93...</td>\n",
       "      <td>2020-11-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>学校办公室关于举办2020年秋季学期校长午餐会的通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/66fc5e810c664b919b...</td>\n",
       "      <td>2020-11-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>中山大学南方学院关于进一步做好疫情防控常态化校园管理工作的通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/c63acd1ca67746ea93...</td>\n",
       "      <td>2020-10-17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>关于国庆中秋假期学生出入校园管理的通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/6e824246669d471ca2...</td>\n",
       "      <td>2020-09-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>校园管理部关于调整班车运行班次的通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/df9afab7eb564161bc...</td>\n",
       "      <td>2020-09-15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>中山大学南方学院关于2020年国庆节、中秋节放假安排的通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/b57e28f6cd5a4cb7b1...</td>\n",
       "      <td>2020-09-11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>中山大学南方学院关于启用电子校园卡的通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/ad7da9ff14494749b7...</td>\n",
       "      <td>2020-09-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>关于开展“节约粮食从点滴做起，请跟我一起行动” 主题团日活动的通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/54573ec40397435ca4...</td>\n",
       "      <td>2020-08-20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>中山大学南方学院关于成立公共管理学院的通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/e28b2797a45d471b88...</td>\n",
       "      <td>2020-08-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>中共中山大学南方学院委员会关于公共管理学系党总支更名的通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/fc277cccd7f440a2bd...</td>\n",
       "      <td>2020-08-05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>校园管理部关于2019～2020学年暑假期间校园生活服务安排的通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/f04e18c7e60b437d8c...</td>\n",
       "      <td>2020-07-24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>中山大学南方学院关于公布2020-2021学年校历的通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/4df100c9d8754356a6...</td>\n",
       "      <td>2020-07-18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>校园管理部关于2020年端午节假期校园生活服务安排的通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/ed13952ecd4343d99d...</td>\n",
       "      <td>2020-06-24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>校园管理部关于2020届毕业生缴纳水费的通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/8eece1bbe3b34e489e...</td>\n",
       "      <td>2020-06-23</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   标题  \\\n",
       "0         校园管理部关于2021年元旦放假校园生活服务安排的通知   \n",
       "1            中山大学南方学院关于2021年元旦放假安排的通知   \n",
       "2                  关于开展2020年知识产权竞赛的通知   \n",
       "3       中山大学南方学院关于举办2020年预防艾滋病巡讲活动的通知   \n",
       "4                  关于开展2020年安全知识竞赛的通知   \n",
       "5   中山大学南方学院关于举办2020～2021学年教职工体育活动的通知   \n",
       "6            中山大学南方学院关于举办2020年体育大会的通知   \n",
       "7          学校办公室关于举办2020年秋季学期校长午餐会的通知   \n",
       "8     中山大学南方学院关于进一步做好疫情防控常态化校园管理工作的通知   \n",
       "9                 关于国庆中秋假期学生出入校园管理的通知   \n",
       "10                 校园管理部关于调整班车运行班次的通知   \n",
       "11      中山大学南方学院关于2020年国庆节、中秋节放假安排的通知   \n",
       "12               中山大学南方学院关于启用电子校园卡的通知   \n",
       "13  关于开展“节约粮食从点滴做起，请跟我一起行动” 主题团日活动的通知   \n",
       "14              中山大学南方学院关于成立公共管理学院的通知   \n",
       "15      中共中山大学南方学院委员会关于公共管理学系党总支更名的通知   \n",
       "16  校园管理部关于2019～2020学年暑假期间校园生活服务安排的通知   \n",
       "17       中山大学南方学院关于公布2020-2021学年校历的通知   \n",
       "18       校园管理部关于2020年端午节假期校园生活服务安排的通知   \n",
       "19             校园管理部关于2020届毕业生缴纳水费的通知   \n",
       "\n",
       "                                                   链接          日期  \n",
       "0   https://www.nfu.edu.cn/tzgg/cd60e06378e5449294...  2020-12-25  \n",
       "1   https://www.nfu.edu.cn/tzgg/16fcbd56eab04220b3...  2020-12-17  \n",
       "2   https://www.nfu.edu.cn/tzgg/155655d4a7e74c7695...  2020-12-16  \n",
       "3   https://www.nfu.edu.cn/tzgg/f381db0e5b3e4746b3...  2020-12-03  \n",
       "4   https://www.nfu.edu.cn/tzgg/ae83ecc6ce894bcb81...  2020-12-03  \n",
       "5   https://www.nfu.edu.cn/tzgg/47ba4c0dfb1443f994...  2020-11-25  \n",
       "6   https://www.nfu.edu.cn/tzgg/21cd39d341924ffd93...  2020-11-07  \n",
       "7   https://www.nfu.edu.cn/tzgg/66fc5e810c664b919b...  2020-11-07  \n",
       "8   https://www.nfu.edu.cn/tzgg/c63acd1ca67746ea93...  2020-10-17  \n",
       "9   https://www.nfu.edu.cn/tzgg/6e824246669d471ca2...  2020-09-25  \n",
       "10  https://www.nfu.edu.cn/tzgg/df9afab7eb564161bc...  2020-09-15  \n",
       "11  https://www.nfu.edu.cn/tzgg/b57e28f6cd5a4cb7b1...  2020-09-11  \n",
       "12  https://www.nfu.edu.cn/tzgg/ad7da9ff14494749b7...  2020-09-08  \n",
       "13  https://www.nfu.edu.cn/tzgg/54573ec40397435ca4...  2020-08-20  \n",
       "14  https://www.nfu.edu.cn/tzgg/e28b2797a45d471b88...  2020-08-05  \n",
       "15  https://www.nfu.edu.cn/tzgg/fc277cccd7f440a2bd...  2020-08-05  \n",
       "16  https://www.nfu.edu.cn/tzgg/f04e18c7e60b437d8c...  2020-07-24  \n",
       "17  https://www.nfu.edu.cn/tzgg/4df100c9d8754356a6...  2020-07-18  \n",
       "18  https://www.nfu.edu.cn/tzgg/ed13952ecd4343d99d...  2020-06-24  \n",
       "19  https://www.nfu.edu.cn/tzgg/8eece1bbe3b34e489e...  2020-06-23  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 输出结果\n",
    "# B-D-1 pd.DataFrame 建构，pandas课有教\n",
    "df = pd.DataFrame( {\n",
    "         \"标题\": parsed.xpath('//div[@class=\"news_title\"]/a/@title'),\n",
    "         \"链接\": list_URL,\n",
    "         \"日期\": parsed.xpath('//font[@class=\"right-more\"]/text()'),\n",
    "     } )\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_excel(\"data_out/nfu_tzgg.xlsx\", sheet_name=\"检索结果\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "200\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n",
      "404\n"
     ]
    }
   ],
   "source": [
    "for i in range(1,100):\n",
    "    r = session.get('https://www.nfu.edu.cn/tzgg/index'+str(i)+'.htm')\n",
    "    print(r.status_code)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "35\n"
     ]
    }
   ],
   "source": [
    "for i in range(1,100):\n",
    "    r = session.get('https://www.nfu.edu.cn/tzgg/index'+str(i)+'.htm')\n",
    "    if r.status_code != 200:\n",
    "        print(i)\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://www.nfu.edu.cn/tzgg/index1.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index2.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index3.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index4.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index5.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index6.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index7.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index8.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index9.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index10.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index11.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index12.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index13.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index14.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index15.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index16.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index17.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index18.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index19.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index20.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index21.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index22.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index23.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index24.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index25.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index26.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index27.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index28.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index29.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index30.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index31.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index32.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index33.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index34.htm']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url_group = ['https://www.nfu.edu.cn/tzgg/index'+str(i)+'.htm' for i in range(1,35)]\n",
    "url_group"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://www.nfu.edu.cn/tzgg/index.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index1.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index2.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index3.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index4.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index5.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index6.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index7.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index8.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index9.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index10.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index11.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index12.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index13.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index14.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index15.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index16.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index17.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index18.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index19.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index20.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index21.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index22.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index23.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index24.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index25.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index26.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index27.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index28.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index29.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index30.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index31.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index32.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index33.htm',\n",
       " 'https://www.nfu.edu.cn/tzgg/index34.htm']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url_group.insert(0,'https://www.nfu.edu.cn/tzgg/index.htm')\n",
    "url_group"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "for url in url_group:\n",
    "    r = session.get(url)\n",
    "#     print(r.html.html)\n",
    "#     设置存档HTML的名称\n",
    "    path = urllib.parse.urlparse(url).path\n",
    "    with open ('html_out/'+path, encoding = \"utf8\", mode = \"w\") as fp:\n",
    "        fp.write(r.html.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# xpath 准备：\n",
    "dict_xpath = {\n",
    "    '链接_xpath':'//div[@class=\"news_title\"]/a/@href',\n",
    "    '标题_xpath':'//div[@class=\"news_title\"]/a/@title',\n",
    "    '日期_xpath':'//font[@class=\"right-more\"]/text()'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pages_content_url(parsed):\n",
    "    list_URL  = [urllib.parse.urlunparse\\\n",
    "                 ([nfu_urlparse.scheme,nfu_urlparse.netloc,'/'+ nfu_urlparse.path.split('/')[1] +'/' + detail_url,'','',''])\\\n",
    "                 for detail_url in parsed.xpath(dict_xpath['链接_xpath'])]\n",
    "    return list_URL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['index.htm', 'index1.htm', 'index10.htm', 'index11.htm', 'index12.htm', 'index13.htm', 'index14.htm', 'index15.htm', 'index16.htm', 'index17.htm', 'index18.htm', 'index19.htm', 'index2.htm', 'index20.htm', 'index21.htm', 'index22.htm', 'index23.htm', 'index24.htm', 'index25.htm', 'index26.htm', 'index27.htm', 'index28.htm', 'index29.htm', 'index3.htm', 'index30.htm', 'index31.htm', 'index32.htm', 'index33.htm', 'index34.htm', 'index35.htm', 'index36.htm', 'index37.htm', 'index38.htm', 'index39.htm', 'index4.htm', 'index40.htm', 'index41.htm', 'index42.htm', 'index43.htm', 'index44.htm', 'index45.htm', 'index46.htm', 'index47.htm', 'index48.htm', 'index49.htm', 'index5.htm', 'index50.htm', 'index51.htm', 'index52.htm', 'index53.htm', 'index54.htm', 'index55.htm', 'index56.htm', 'index57.htm', 'index58.htm', 'index59.htm', 'index6.htm', 'index60.htm', 'index61.htm', 'index62.htm', 'index63.htm', 'index64.htm', 'index65.htm', 'index66.htm', 'index67.htm', 'index68.htm', 'index69.htm', 'index7.htm', 'index70.htm', 'index71.htm', 'index72.htm', 'index73.htm', 'index74.htm', 'index75.htm', 'index76.htm', 'index77.htm', 'index78.htm', 'index79.htm', 'index8.htm', 'index80.htm', 'index81.htm', 'index82.htm', 'index83.htm', 'index84.htm', 'index85.htm', 'index86.htm', 'index87.htm', 'index88.htm', 'index89.htm', 'index9.htm', 'index90.htm']\n"
     ]
    }
   ],
   "source": [
    "import os     # 读取html文件\n",
    "\n",
    "list_df = []\n",
    "\n",
    "\n",
    "files= os.listdir('html_out/tzgg/')\n",
    "print(files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['index.htm', 'index1.htm', 'index10.htm', 'index11.htm', 'index12.htm', 'index13.htm', 'index14.htm', 'index15.htm', 'index16.htm', 'index17.htm', 'index18.htm', 'index19.htm', 'index2.htm', 'index20.htm', 'index21.htm', 'index22.htm', 'index23.htm', 'index24.htm', 'index25.htm', 'index26.htm', 'index27.htm', 'index28.htm', 'index29.htm', 'index3.htm', 'index30.htm', 'index31.htm', 'index32.htm', 'index33.htm', 'index34.htm', 'index35.htm', 'index36.htm', 'index37.htm', 'index38.htm', 'index39.htm', 'index4.htm', 'index40.htm', 'index41.htm', 'index42.htm', 'index43.htm', 'index44.htm', 'index45.htm', 'index46.htm', 'index47.htm', 'index48.htm', 'index49.htm', 'index5.htm', 'index50.htm', 'index51.htm', 'index52.htm', 'index53.htm', 'index54.htm', 'index55.htm', 'index56.htm', 'index57.htm', 'index58.htm', 'index59.htm', 'index6.htm', 'index60.htm', 'index61.htm', 'index62.htm', 'index63.htm', 'index64.htm', 'index65.htm', 'index66.htm', 'index67.htm', 'index68.htm', 'index69.htm', 'index7.htm', 'index70.htm', 'index71.htm', 'index72.htm', 'index73.htm', 'index74.htm', 'index75.htm', 'index76.htm', 'index77.htm', 'index78.htm', 'index79.htm', 'index8.htm', 'index80.htm', 'index81.htm', 'index82.htm', 'index83.htm', 'index84.htm', 'index85.htm', 'index86.htm', 'index87.htm', 'index88.htm', 'index89.htm', 'index9.htm', 'index90.htm']\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>标题</th>\n",
       "      <th>链结</th>\n",
       "      <th>日期</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>校园管理部关于2021年元旦放假校园生活服务安排的通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/cd60e06378e5449294...</td>\n",
       "      <td>2020-12-25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>中山大学南方学院关于2021年元旦放假安排的通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/16fcbd56eab04220b3...</td>\n",
       "      <td>2020-12-17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>关于开展2020年知识产权竞赛的通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/155655d4a7e74c7695...</td>\n",
       "      <td>2020-12-16</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>中山大学南方学院关于举办2020年预防艾滋病巡讲活动的通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/f381db0e5b3e4746b3...</td>\n",
       "      <td>2020-12-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>关于开展2020年安全知识竞赛的通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/ae83ecc6ce894bcb81...</td>\n",
       "      <td>2020-12-03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>562</th>\n",
       "      <td>2</td>\n",
       "      <td>“南苑青年”系列讲座之第十三讲的通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/bbd14d55a99247a79f...</td>\n",
       "      <td>2015-04-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>563</th>\n",
       "      <td>3</td>\n",
       "      <td>中山大学南方学院关于举办“南方湖畔·艺彩纷呈”第七届校园文化艺术节活动通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/a8e5e752e409486da2...</td>\n",
       "      <td>2015-04-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>564</th>\n",
       "      <td>4</td>\n",
       "      <td>学院办公室关于2015年五一放假安排的通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/e3f763049ee54cfc8c...</td>\n",
       "      <td>2015-04-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>565</th>\n",
       "      <td>5</td>\n",
       "      <td>中山大学南方学院关于2015年公共机构节能宣传作品征集活动的通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/f3ae1aa3ccdb4d87bc...</td>\n",
       "      <td>2015-04-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>566</th>\n",
       "      <td>6</td>\n",
       "      <td>关于开展校园网络和运营商移动网络使用情况调查的通知</td>\n",
       "      <td>https://www.nfu.edu.cn/tzgg/6de44f6a618540ef82...</td>\n",
       "      <td>1970-01-01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>687 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     index                                     标题  \\\n",
       "0        0            校园管理部关于2021年元旦放假校园生活服务安排的通知   \n",
       "1        1               中山大学南方学院关于2021年元旦放假安排的通知   \n",
       "2        2                     关于开展2020年知识产权竞赛的通知   \n",
       "3        3          中山大学南方学院关于举办2020年预防艾滋病巡讲活动的通知   \n",
       "4        4                     关于开展2020年安全知识竞赛的通知   \n",
       "..     ...                                    ...   \n",
       "562      2                     “南苑青年”系列讲座之第十三讲的通知   \n",
       "563      3  中山大学南方学院关于举办“南方湖畔·艺彩纷呈”第七届校园文化艺术节活动通知   \n",
       "564      4                  学院办公室关于2015年五一放假安排的通知   \n",
       "565      5       中山大学南方学院关于2015年公共机构节能宣传作品征集活动的通知   \n",
       "566      6              关于开展校园网络和运营商移动网络使用情况调查的通知   \n",
       "\n",
       "                                                    链结          日期  \n",
       "0    https://www.nfu.edu.cn/tzgg/cd60e06378e5449294...  2020-12-25  \n",
       "1    https://www.nfu.edu.cn/tzgg/16fcbd56eab04220b3...  2020-12-17  \n",
       "2    https://www.nfu.edu.cn/tzgg/155655d4a7e74c7695...  2020-12-16  \n",
       "3    https://www.nfu.edu.cn/tzgg/f381db0e5b3e4746b3...  2020-12-03  \n",
       "4    https://www.nfu.edu.cn/tzgg/ae83ecc6ce894bcb81...  2020-12-03  \n",
       "..                                                 ...         ...  \n",
       "562  https://www.nfu.edu.cn/tzgg/bbd14d55a99247a79f...  2015-04-08  \n",
       "563  https://www.nfu.edu.cn/tzgg/a8e5e752e409486da2...  2015-04-07  \n",
       "564  https://www.nfu.edu.cn/tzgg/e3f763049ee54cfc8c...  2015-04-01  \n",
       "565  https://www.nfu.edu.cn/tzgg/f3ae1aa3ccdb4d87bc...  2015-04-01  \n",
       "566  https://www.nfu.edu.cn/tzgg/6de44f6a618540ef82...  1970-01-01  \n",
       "\n",
       "[687 rows x 4 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "list_df = []\n",
    "\n",
    "# os listdir取出某文件夹中的文件的名称组成列表\n",
    "files= os.listdir('html_out/tzgg/')\n",
    "print(files)\n",
    "\n",
    "for html in files:\n",
    "    with open('html_out/tzgg/'+html,encoding='utf8',mode='r') as fp:\n",
    "        html_load = fp.read()\n",
    "        parsed = requests_html.soup_parse(html_load)\n",
    "        list_URL = pages_content_url(parsed)\n",
    "        \n",
    "        df = pd.DataFrame( {\n",
    "         \"标题\": parsed.xpath(dict_xpath['标题_xpath']),\n",
    "         \"链结\": list_URL,\n",
    "         \"日期\": parsed.xpath(dict_xpath['日期_xpath']),\n",
    "        } )\n",
    "        list_df.append(df)\n",
    "\n",
    "\n",
    "# 凭借表格pandas 方法concat\n",
    "# 数据分析的方法 concat拼接DataFrame  sort_values排序\n",
    "df_all = pd.concat(list_df).reset_index().sort_values(by='日期',ascending=False)\n",
    "display(df_all)    \n",
    "\n",
    "with pd.ExcelWriter('data_out/nfu_通知公告.xlsx',mode='w',engine=\"openpyxl\") as writer:  \n",
    "            df_all.to_excel(writer, sheet_name='通知公告')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
